'''gaoyunzhi@gmail.com
1:09 PM 10/15/2012
get saved webpage, if not saved, save the webpage before doing this
input: webpage link
output: webpage as string
mutation: if the webpage is not saved, save it
'''

from toFname import toFname
from sysPath import folderPath
import urllib2
import os
import time,sys

def getWebpage(link,dataDir='webpages',silent=True, timeSleep=0, \
               cookies=None, reLoad=False):
    if not silent: print 'link=', link
    dataDir=folderPath(dataDir) # change path according to the system
    fname=toFname(link)
    if not silent:
        print 'save in '+dataDir+fname
    try:
        if reLoad==True:
            raise Exception('') # if reload if true go to except part
        f=open(dataDir+fname,'r')
        if not silent: print 'read from cached file'
        page=f.read()
        f.close()
    except:
        if not silent: print 'read from web'
        if dataDir!='' and (not os.path.exists(dataDir)):os.makedirs(dataDir)        
        for i in range(10):
            try:
                page_info = urllib2.build_opener()
                #browser=['Mozilla/5.0']
                page_info.addheaders = [('User1', 'safari/536.25')]
                if cookies==None:
                    #print 'no cookie'
                    cookies=''
                #print cookies
                page_info.addheaders.append(('Cookie', cookies))
                page = page_info.open(link)
                page=page.read()
                break
            except urllib2.HTTPError, e:
                print e.code
                page=''
            except urllib2.URLError, e:
                print e.args
                page=''
            time.sleep(timeSleep)
        if page=='':
            return
        if os.path.exists(dataDir+fname):
            fileMTime=os.path.getmtime(dataDir+fname)
            newFileName=dataDir+toFname(link,hasHtml=False)+str(fileMTime)+'.html'
            os.renames(dataDir+fname, newFileName)
            
            
        f=open(dataDir+fname,'w')
        f.write(page)
        f.close()
    return page

def test():

    cookies='1st=269B87621B53593C; RES_TRACKINGID=231516832252964; __utmz=17384120.1352718169.7.2.utmcsr=stackoverflow.com|utmccn=(referral)|utmcmd=referral|utmcct=/questions/13312499/is-it-possible-to-fetch-data-from-a-javascript-based-website; peapodCookie20=-f4e6753%3a13af41d7537%3a-63e8ae0a1eFKJ; JSESSIONID=NORNM5YK2JTJQCQBD0WSIIQ; stype=1; __utma=17384120.3768722719745875500.1352418522.1352747370.1352750502.9; __utmc=17384120; __utmb=17384120.20.10.1352750502; RES_SESSIONID=317009223159402; ResonanceSegment=2'
    print getWebpage('http://www.peapod.com/itemDetailView.jhtml?productId=155508',reLoad=True,\
                     silent=False, cookies=cookies)


if __name__=='__main__':
    print 'contains getWegpage method'
    print 'get saved webpage, if not saved, save the webpage before doing this'
    print 'some tests'
    test()
    
